package xin.ganbo.crawler.start;

import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Spider;
import xin.ganbo.crawler.processor.CsdnPageProcessor;

import java.util.ArrayList;
import java.util.List;

/**
 * 拉取csdn博客文章启动器
 * @author ganbo E-mail:bo.gan@56qq.com
 * @since 2017-11-30 15:03
 */
@Component
public class CsdnProcessorStart {

    public CsdnProcessorStart(){
        List<String> names = new ArrayList<>();
        names.add("yerenyuan_pku");
        names.add("tgbblog");
        names.add("myjs999");
        names.add("notbaron");

        List<String> urls = new ArrayList<>();
        for(String name:names){
            urls.add("http://blog.csdn.net/"+name);
        }
        long startTime, endTime;
        System.out.println("开始爬取...");
        startTime = System.currentTimeMillis();
        String[] urlss = new String[urls.size()];
        Spider.create(new CsdnPageProcessor()).addUrl(urls.toArray(urlss)).thread(5).run();
        endTime = System.currentTimeMillis();
        System.out.println("爬取结束，耗时约" + ((endTime - startTime) / 1000) + "秒。。。");
    }

}
